package sparkstreaming.eighteenthday1.lesson2

import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.streaming.{Seconds, StreamingContext}

object HDFSWordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName(s"${this.getClass.getSimpleName}")
    val sc = new SparkContext(conf)

    val ssc = new StreamingContext(sc, Seconds(5))
    //val ssc = new StreamingContext(conf, Seconds(5))

    val fileDStream: DStream[String] = ssc.textFileStream("hdfs://hadoop01:9000/sparkstreaming")

    val WordCountDStream: DStream[(String, Int)] = fileDStream.flatMap(_.split(",")).map((_,1)).reduceByKey(_+_)

    WordCountDStream.print()

    ssc.start()
    ssc.awaitTermination()
    ssc.stop()
  }

}
